import re
import json

def clean_tags(tag_str):
    # 1. 清洗标签字符串
    tag_str = tag_str.replace('\r', '').replace('\n', '').strip()
    tags = [t.strip() for t in tag_str.split(',') if t.strip() != '']

    # 2. 去除纯英文标签（保留混合的如“KTV包间”）
    tags = [t for t in tags if not re.fullmatch(r'[A-Za-z\-_\.\s]+', t)]

    # 3. 去重
    tags = list(dict.fromkeys(tags))

    if len(tags) > 6:
        tags = tags[:6]  # 多于7个，只保留前7个

    return ",".join(tags)

# 示例：清洗整个 JSON 列表
def clean_json_dataset(data_list):
    cleaned_data = []
    for item in data_list:
        cleaned_tag = clean_tags(item.get("tag", ""))
        if cleaned_tag is not None:
            item["tag"] = cleaned_tag
            cleaned_data.append(item)
    return cleaned_data

# 读取原始 JSON 文件并保存清洗后的版本
def process_json_file(input_path, output_path):
    with open(input_path, 'r', encoding='utf-8') as f:
        data = json.load(f)

    cleaned = clean_json_dataset(data)

    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(cleaned, f, ensure_ascii=False, indent=2)
    print(f"[INFO] 清洗完成！原始样本: {len(data)} → 保留样本: {len(cleaned)}")

# 使用方法示例
process_json_file("processed_NewGZ_tag/test/data_tag_test.json", "processed_NewGZ_tag/test/cleaned_test.json")
